Now that I've streamlined the MCMC process, I am going to submit multiple chains simultaneously. This notebook will make multiple, similar config files, for broad comparison.

This may be rolled into pearce as a helper function, I haven't decided.

Right now I'm interested in running chain in each of the 35 boxes, all at

The xi_gg & xi_gm analysis, neglecting covariance
The same, at fixed HOD
Full analysis just for xi
Full analysis just for xi_gm



In [2]:

    
import yaml
import copy
from os import path



In [3]:

    
orig_cfg_fname = '/home/users/swmclau2/Git/pearce/bin/mcmc/nh_gg_mcmc_config.yaml'
with open(orig_cfg_fname, 'r') as yamlfile:
    orig_cfg = yaml.load(yamlfile)



In [4]:

    
orig_cfg









    Out[4]:





{'chain': {'fixed_params': {},
  'nburn': 0,
  'nsteps': 10000,
  'nwalkers': 400,
  'seed': 0},
 'data': {'cov': {'emu_cov_fname': '/home/users/swmclau2/Git/pearce/bin/covmat/xi_gg_nh_emu_cov.npy',
   'meas_cov_fname': '/home/users/swmclau2/Git/pearce/notebooks/meas_cov_testboxes_gg.npy',
   'shot': True},
  'obs': {'mean': False,
   'obs': 'xi',
   'rbins': [0.07943282,
    0.11220185,
    0.15848932,
    0.22387211,
    0.31622777,
    0.44668359,
    0.63095734,
    0.89125094,
    1.25892541,
    1.77827941,
    2.51188643,
    3.54813389,
    5.01187234,
    7.07945784,
    10.0,
    14.12537545,
    19.95262315,
    28.18382931,
    39.81071706]},
  'sim': {'gal_type': 'HOD',
   'hod_name': 'zheng07',
   'hod_params': {'alpha': 1.083,
    'logM0': 13.2,
    'logM1': 14.2,
    'sigma_logM': 0.2},
   'nd': '5e-4',
   'scale_factor': 1.0,
   'sim_hps': {'boxno': 3,
    'downsample_factor': '1e-2',
    'particles': False,
    'realization': 2,
    'system': 'sherlock'},
   'simname': 'testbox'}},
 'emu': {'emu_hps': {},
  'emu_type': 'NashvilleHot',
  'fixed_params': {'z': 0.0},
  'training_file': '/home/users/swmclau2/scratch/xi_gg_zheng07_cosmo_v3/PearceXiggCosmo.hdf5'},
 'fname': '/scratch/users/swmclau2/PearceMCMC/pearce_mcmc_nh_gg_v6.hdf5'}

orig_sbatch_fname = '/home/users/swmclau2/Git/pearce/bin/mcmc/config/pearce_mcmc_config_test.sbatch' with open(orig_sbatch_fname, 'r') as f: lines = [] for line in f: lines.append(line) orig_sbatch = ''.join(lines)



In [5]:

    
#this will enable easier string formatting
sbatch_template = """#!/bin/bash
#SBATCH --job-name={jobname}
#SBATCH --time=08:00:00
#SBATCH -p iric
#SBATCH -o /home/users/swmclau2/Git/pearce/bin/mcmc/config/{jobname}.out
#SBATCH --ntasks=16
#SBATCH --exclusive

module load python/2.7.13
module load py-scipystack
module load hdf5/1.10.0p1
python /home/users/swmclau2/Git/pearce/pearce/inference/initialize_mcmc.py {jobname}.yaml
python /home/users/swmclau2/Git/pearce/pearce/inference/run_mcmc.py {jobname}.yaml
"""

$$ \xi_{gg} \& \xi_{gm} $$

For this one, only have to change boxno and realization no.

tmp_cfg = copy.deepcopy(orig_cfg) directory = "/home/users/swmclau2/Git/pearce/bin/mcmc/config/" output_dir = "/home/users/swmclau2/scratch/PearceMCMC/" jobname_template = "b{boxno}r{realization}_xi_gg_gm_joint_ind" for boxno in xrange(7): for realization in xrange(5): tmp_cfg['data']['sim']['sim_hps']['boxno'] = boxno tmp_cfg['data']['sim']['sim_hps']['realization'] = realization tmp_cfg['emu']['emu_type'] = ['SpicyBuffalo', 'SpicyBuffalo'] tmp_cfg['emu']['training_file'] = ['/scratch/users/swmclau2/xi_zheng07_cosmo_lowmsat/PearceRedMagicXiCosmoFixedNd.hdf5',\ '/scratch/users/swmclau2/xi_gm_cosmo/PearceRedMagicXiGMCosmoFixedNd.hdf5'] jobname = jobname_template.format(boxno=boxno, realization=realization) tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5') with open(path.join(directory, jobname +'.yaml'), 'w') as f: yaml.dump(tmp_cfg, f) with open(path.join(directory, jobname + '.sbatch'), 'w') as f: f.write(sbatch_template.format(jobname=jobname))

%%bash ls /home/users/swmclau2/Git/pearce/bin/mcmc/config/*

$$ \xi_{gg}$$, Fixed HOD

Same as above, but also adjust fixed parameters.

tmp_cfg = copy.deepcopy(orig_cfg) directory = "/home/users/swmclau2/Git/pearce/bin/mcmc/config/" jobname_template = "b{boxno}r{realization}_xi_gg_gm_joint_ind_fixed_HOD" output_dir = "/home/users/swmclau2/scratch/PearceMCMC/" for boxno in xrange(7): for realization in xrange(5): tmp_cfg['data']['sim']['sim_hps']['boxno'] = boxno tmp_cfg['data']['sim']['sim_hps']['realization'] = realization tmp_cfg['chain']['fixed_params'] = 'HOD' tmp_cfg['emu']['emu_type'] = ['SpicyBuffalo', 'SpicyBuffalo'] tmp_cfg['emu']['training_file'] = ['/scratch/users/swmclau2/xi_zheng07_cosmo_lowmsat/PearceRedMagicXiCosmoFixedNd.hdf5',\ '/scratch/users/swmclau2/xi_gm_cosmo/PearceRedMagicXiGMCosmoFixedNd.hdf5'] jobname = jobname_template.format(boxno=boxno, realization=realization) tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5') with open(path.join(directory, jobname +'.yaml'), 'w') as f: yaml.dump(tmp_cfg, f) with open(path.join(directory, jobname + '.sbatch'), 'w') as f: f.write(sbatch_template.format(jobname=jobname))

$$ \xi_{gg}$$

Have to change a few of the observations here.

tmp_cfg = copy.deepcopy(orig_cfg) directory = "/home/users/swmclau2/Git/pearce/bin/mcmc/config/" output_dir = "/home/users/swmclau2/scratch/PearceMCMC/" jobname_template = "b{boxno}r{realization}_xi_gg" #for boxno in xrange(7): boxno = 3 for realization in xrange(5): tmp_cfg['data']['sim']['sim_hps']['boxno'] = boxno tmp_cfg['data']['sim']['sim_hps']['realization'] = realization #tmp_cfg['data']['cov']['emu_cov_fname'] = 'xigg_scov.npy' #tmp_cfg['data']['obs']['obs'] = 'xi' jobname = jobname_template.format(boxno=boxno, realization=realization) tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5') with open(path.join(directory, jobname +'.yaml'), 'w') as f: yaml.dump(tmp_cfg, f) with open(path.join(directory, jobname + '.sbatch'), 'w') as f: f.write(sbatch_template.format(jobname=jobname))

$$ \xi_{gg}$$ + AB

Have to change a few of the observations here.



In [6]:

    
tmp_cfg = copy.deepcopy(orig_cfg)
directory = "/home/users/swmclau2/Git/pearce/bin/mcmc/config/"
output_dir = "/home/users/swmclau2/scratch/PearceMCMC/"
jobname_template = "b{boxno}r{realization}_xi_gg_hsab"
#for boxno in xrange(7):
boxno = 3
for realization in xrange(5):
    tmp_cfg['data']['sim']['sim_hps']['boxno'] = boxno
    tmp_cfg['data']['sim']['sim_hps']['realization'] = realization

    #tmp_cfg['data']['cov']['emu_cov_fname'] = 'xigg_scov.npy'

    #tmp_cfg['data']['obs']['obs'] = 'xi'
    tmp_cfg['sim']= {'gal_type': 'HOD',
   'hod_name': 'hsabZheng07',
   'hod_params': {'alpha': 1.083,
    'logM0': 13.2,
    'logM1': 14.2,
    'sigma_logM': 0.2,
    'mean_occupation_cens_param1':0.6,
    'mean_occupation_sats_param1':-0.4},
   'nd': '5e-4',
   'scale_factor': 1.0,
   'sim_hps': {'boxno': 3,
    'downsample_factor': '1e-2',
    'particles': False,
    'realization': 2,
    'system': 'sherlock'},
   'simname': 'testbox'}

    jobname = jobname_template.format(boxno=boxno, realization=realization)
    tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5')

    with open(path.join(directory, jobname +'.yaml'), 'w') as f:
        yaml.dump(tmp_cfg, f)

    with open(path.join(directory, jobname + '.sbatch'), 'w') as f:
        f.write(sbatch_template.format(jobname=jobname))

$$ \xi_{gm}$$

Have to change a few of the observations here.

tmp_cfg = copy.deepcopy(orig_cfg) directory = "/home/users/swmclau2/Git/pearce/bin/mcmc/config/" output_dir = "/home/users/swmclau2/scratch/PearceMCMC/" jobname_template = "b{boxno}r{realization}_xi_gm" for boxno in xrange(7): for realization in xrange(5): tmp_cfg['data']['sim']['sim_hps']['boxno'] = boxno tmp_cfg['data']['sim']['sim_hps']['realization'] = realization tmp_cfg['data']['cov']['emu_cov_fname'] = 'xigm_scov.npy' tmp_cfg['data']['obs']['obs'] = 'xi_gm' tmp_cfg['emu']['emu_type'] = 'SpicyBuffalo' tmp_cfg['emu']['training_file'] = '/scratch/users/swmclau2/xi_gm_cosmo/PearceRedMagicXiGMCosmoFixedNd.hdf5' jobname = jobname_template.format(boxno=boxno, realization=realization) tmp_cfg['fname'] = path.join(output_dir, jobname+'.hdf5') with open(path.join(directory, jobname +'.yaml'), 'w') as f: yaml.dump(tmp_cfg, f) with open(path.join(directory, jobname + '.sbatch'), 'w') as f: f.write(sbatch_template.format(jobname=jobname))

%%bash ls /home/users/swmclau2/Git/pearce/bin/mcmc/config/*.yaml



In [ ]: